/*
 * Copyright (c) 2024 Huawei Technologies Co.,Ltd.
 *
 * openGauss is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *
 * http://license.coscl.org.cn/MulanPSL2
 *
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
 * ---------------------------------------------------------------------------------------
 *
 * imcs_insert.h
 * routines to support ColStore
 *
 *
 * IDENTIFICATION
 * src/include/access/imcs/imcs_insert.h
 *
 * ---------------------------------------------------------------------------------------
 */

#ifndef IMCS_INSERT_H
#define IMCS_INSERT_H

#include "imcs_am.h"
#include "access/cstore_delta.h"
#include "imcs_vector.h"
#include "imcs_minmax_func.h"
#include "storage/cstore/cstore_compress.h"
#include "storage/spin.h"
#include "imcs_ctlg.h"
#include "catalog/gs_imcs.h"

struct IMInsertArg {
    /* map to IMCSInsert::m_tmpBatchRows.
     *
     * because it's a temp batchrows, so
     * 1. it's used by each batch inserter when PCK exists, or
     * 2. it's shared by each partition when inserting into partitioned table,
     */
    IMCSBulkloadRows *tmp_batch_rows;

    /* map to IMCSInsert::m_idxBatchRow
     *
     * because it's a temp batchrows, so
     * 1. it's shared by each index inserter when inserting into ordinary table, or
     * 2. it's shared by each index inserter of each partition when inserting into
     * partitioned table.
     */
    IMCSBulkloadRows *idx_batch_row;

    ResultRelInfo *es_result_relations;

    /* sort type: tuple sorting or batch sorting */

    /* which function will be called,
     * 1. false => void IMCSInsert::BatchInsert(IMCSBulkloadRows *batch_row_ptr, int options)
     * 2. true  => void IMCSInsert::BatchInsert(VectorBatch *pBatch,        int options)
     * this value must be defined explicitly.
     */
    bool using_vectorbatch;

    IMInsertArg()
    {
        tmp_batch_rows = NULL;
        idx_batch_row = NULL;
        es_result_relations = NULL;
        using_vectorbatch = true; /* caller should specify the right way */
    }
};

/*
 * This class provides some API for batch insert in ColStore.
 * In future, we can add more API.
 */
class IMCSInsert : public BaseObject {
public:
    // IMCSInsert(_in_ Relation relation, _in_ const IMInsertArg &args, _in_ bool is_update_cu, _in_ Plan *plan,
    //              _in_ ImcsMemInfoArg *argmem_info); not used
    IMCSInsert(_in_ Relation relation, _in_ ResultRelInfo *result_rel_info, _in_ bool is_update_cu, _in_ Plan *plan,
        _in_ ImcsMemInfoArg *argmem_info, TupleDesc tupdesc_with_ctid);

    virtual ~IMCSInsert();
    virtual void Destroy();

    void BeginBatchInsert();
    void EndBatchInsert();

    /*
     * Batch insert interface
     */
    void BatchInsert(IMCSBulkloadRows *batch_row_ptr, int options);
    void BatchInsert(VectorBatch *pVec, int options);
    void AppendImcus(IMCU *old_cu, IMCUDesc *old_cu_desc, IMCU *new_cu, IMCUDesc *new_cu_desc,
        bool &use_exist_imcu_cache_desc, int col_idx);
    void BatchInsertCommon(IMCSBulkloadRows *batch_row_ptr, IMCU **old_cus = nullptr,
        IMCUDesc **old_cus_descs = nullptr, bool multithread = false, IMCSDesc *imcs_desc = NULL);
    void UpdateAppendedImcuBuf(IMCU *old_cu, IMCU *new_cu, IMCUDesc *old_desc, IMCUDesc *new_desc,
        bool is_old_desc_null_cu);
    void UpdateAppendedImcuDesc(IMCUDesc *old_cu_desc, IMCUDesc *new_cu_desc, int col_idx);
    void UpdateAppendedImcu(IMCU *dest_cu, IMCU *src_cu, IMCUDesc *dest_cu_desc, IMCUDesc *src_cu_desc,
        bool is_old_desc_null_cu);
    void CopyToCuBm(IMCU *dest_cu, int offset_in_dest_bm, IMCU *src_cu, int raw_cnt_in_src, bool src_null_cu,
        bool src_has_null);
    void FlashData(int options);

    void CUInsert(_in_ IMCSBatchCUData *cu_data, _in_ int options);

    void SetEndFlag();

    inline bool IsEnd()
    {
        return m_insert_end_flag;
    }

    static void InitInsertArg(Relation rel, ResultRelInfo *result_rel_info, bool using_vectorbatch, IMInsertArg &args);
    static void DeInitInsertArg(IMInsertArg &args);
    static void InitIndexInsertArg(Relation heap_rel, const int *keys_map, int nkeys, IMInsertArg &args);
    void InitInsertMemArg(Plan *plan, ImcsMemInfoArg *argmem_info);


    inline MemoryContext GetTmpMemCnxt()
    {
        return m_tmpMemCnxt;
    }

    inline IMCSBulkloadRows *GetBufferedBatchRows()
    {
        return m_bufferedBatchRows;
    }

    void SortAndInsert(int options);

    Relation m_relation;
    TupleDesc td_tid; // TupleDesc with ctid
    IMCU ***m_aio_cu_pptr;
    AioDispatchCUDesc_t ***m_aio_dispath_cudesc;
    int *m_aio_dispath_idx;
    uint32 m_cur_cuid;

    slock_t populate_mutex;

    /* memory info for memory adjustment */
    ImcsMemInfoArg *m_cstorInsertMem;
    CacheSlotId_t get_slot_id()
    {
        return m_slot_id;
    }

private:
    void FreeMemAllocateByAdio();
    /* Whether need partial clustering when load
     */
    inline bool NeedPartialSort(void) const;

    /* Write IMCU data and IMCUDesc
     */
    void SaveAll(int options, const char *del_bitmap = NULL);

    void form_cu_desc_info(int options, const char *del_bitmap = NULL);

    void CUListFlushAll(int attno);
    void CUListWriteCompeleteIO(int col, int count);

    void CUWrite(int attno, int col);
    void CUListWrite();

    /* Compress batch rows into IMCU
     * Get min/max of IMCU
     */
    IMCU *FormCU(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, TupleDesc tupdesc_with_ctid,
        bool compress);
    Size FormCUTInitMem(IMCU *cu_ptr, IMCSBulkloadRows *batch_row_ptr, int col, bool has_null);
    void FormCUTCopyMem(IMCU *cu_ptr, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, Size dtSize, int col,
        bool has_null);
    template <bool hasNull> void FormCUT(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr);
    template <bool hasNull>
    void FormCUTNumeric(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr);
    template <bool hasNull>
    void FormCUTNumString(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr);

    template <bool bpcharType, bool hasNull, bool has_MinMax_func>
    bool FormNumberStringCU(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr);

    template <bool hasNull>
    bool TryFormNumberStringCU(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr,
        uint32 atttypid);

    void InitIndexColId(int which_index);

    void InitIndexInfo();

    void InitDeltaInfo();

    /* Insert delta table */
    void InsertDeltaTable(IMCSBulkloadRows *batch_row_ptr, int options);

    /* Insert index table
     */
    void InsertIdxTableIfNeed(IMCSBulkloadRows *batch_row_ptr, uint32 cu_id);

    void InsertNotPsortIdx(int indice);

    void FlushIndexDataIfNeed();

    void InitFuncPtr();

    void InitColSpaceAlloc();

    bool TryEncodeNumeric(int col, IMCSBulkloadRows *batch_row_ptr, IMCUDesc *cu_desc_ptr, IMCU *cu_ptr, bool has_null);
    void DoBatchInsert(int options);

    typedef void (IMCSInsert::*m_formCUFunc)(int, IMCSBulkloadRows *, IMCUDesc *, IMCU *);
    struct FormCUFuncArray {
        m_formCUFunc col_form_cu[2];
    };
    inline void SetFormCUFuncArray(Form_pg_attribute attr, int col);

    bool m_isUpdate;
    /* Function Pointer Array Area */
    ImcsFuncSetMinMax *m_setMinMaxFuncs; /* min/max value function */
    FormCUFuncArray *m_formCUFuncArray;  /* Form IMCU function pointer */

    /* If relation has cluster key, it will work for partial sort */

    IMCUDesc **m_cuDescPPtr;               /* The cudesc of all columns of m_relation */
    IMCU **m_cuPPtr;                       /* The IMCU of all columns of m_relation; */
    IMCUStorage **m_cuStorage;             /* IMCU storage */
    compression_options *m_cuCmprsOptions; /* compression filter */
    cu_tmp_compress_info m_cuTempInfo;     /* temp info for IMCU compression */

    /* buffered batchrows for many VectorBatch values */
    IMCSBulkloadRows *m_bufferedBatchRows;

    /* temp batchrows for fetching values from sort processor */
    IMCSBulkloadRows *m_tmpBatchRows;

    /* memory context for avoiding memory leaks during bulk-insert */
    MemoryContext m_tmpMemCnxt;
    MemoryContext m_batchInsertCnxt;

    /* work for delta insert */
    Relation m_delta_relation;
    TupleDesc m_delta_desc;

    /* work for Index insert */
    ResultRelInfo *m_resultRelInfo; /* contain index meta info */

    Relation *m_idxRelation;         /* index relations */
    Relation *m_deltaIdxRelation;    /* delta index relations */
    IMInsertArg *m_idxInsertArgs;    /* index inserting arguments */
    IMCSInsert **m_idxInsert;        /* index inserter */
    int **m_idxKeyAttr;              /* index keys */
    int *m_idxKeyNum;                /* index keys' number */
    IMCSBulkloadRows *m_idxBatchRow; /* shallow copy for index insert */
    EState *m_estate;                /* estate information */
    ExprContext *m_econtext;         /* curr context for expr eval */
    Datum *m_fake_values;            /* value array to form fake heap tuple */
    bool *m_fake_isnull;             /* null array to from fake heap tuple */

    /* ADIO info */
    MemoryContext m_aio_memcnxt;
    int32 *m_aio_cache_write_threshold;
    File **m_vfdList;

    /* the max number of values within one IMCU */
    int m_fullCUSize;

    /* the number delta threshold */
    int m_delta_rows_threshold;

    /* compression options. */
    int16 m_compress_modes;

    /* indicate the end of insert or not */
    bool m_insert_end_flag;

    CacheSlotId_t m_slot_id;
};

typedef struct PopulateShared {
    /*
     * These fields are not modified during the sort.  They primarily exist
     * for the benefit of worker processes that need to create BTSpool state
     * corresponding to that used by the leader.
     */
    Oid heaprelid;
    Oid indexrelid;
    Oid heappartid;
    Oid indexpartid;
    int scantuplesortstates;

    /*
     * mutex protects all fields before heapdesc.
     *
     * These fields contain status information of interest to B-Tree index
     * builds that must work just the same when an index is built in parallel.
     */
    slock_t mutex;

    /*
     * Mutable state that is maintained by workers, and reported back to
     * leader at end of parallel scan.
     *
     * reltuples is the total number of input heap tuples.
     *
     * havedead indicates if RECENTLY_DEAD tuples were encountered during
     * build.
     *
     * indtuples is the total number of tuples that made it into the index.
     *
     * brokenhotchain indicates if any worker detected a broken HOT chain
     * during build.
     */
    double reltuples;
    double *alltuples;
    uint32 nparts;
    bool brokenhotchain;
    int workmem[2];
    pg_atomic_uint64 curiter;

    /*
     * This variable-sized field must come last.
     */
    ParallelHeapScanDescData heapdesc;
    int2 *imcs_key_need_populate;
    int imcs_need_populate_nkeys;
    Form_gs_imcs gsimcs_form;
    TupleDesc tupdesc;
    // IMInsertArg insertArgs;
    ResultRelInfo *result_rel_info;
    int32 m_cur_cuid_parallel;
    Relation rel;
    bool is_partition_table;
    bool need_populate_ctid;
    uint32 cur_smp_id;
    LWLock *parallel_populate_lock;
    uint32 parallel_populate_gap;
    uint32 total_pages;
} PopulateShared;


/*
 * Status for leader in parallel index build.
 */
typedef struct PopulateLeader {
    /*
     * nparticipanttuplesorts is the exact number of worker processes
     * successfully launched, plus one leader process if it participates as a
     * worker (only DISABLE_LEADER_PARTICIPATION builds avoid leader
     * participating as a worker).
     */
    int nparticipants;

    /*
     * Leader process convenience pointers to shared state (leader avoids TOC
     * lookups).
     *
     */
    PopulateShared *bulkload_shared;
} PopulateLeader;


/* Working state for btbuild and its callback */
typedef struct {
    Relation heap_rel;
    /*
     * btleader is only present when a parallel index build is performed, and
     * only in the leader process. (Actually, only the leader has a
     * BTBuildState.  Workers have their own spool and spool2, though.)
     */
    PopulateLeader *leader;
} PopulateState;

#endif /* IMCS_INSERT_H */
